Correlated Features Simulation
We will first investigate how various feature importance methods treat correlated variables under different simulation scenarios.
n <- 500
p <- 10
beta <- 1
noise_sd <- 1
data_ls <- list()
vimp_ls <- list()
fit_ls <- list()
# case 1: independent features
X <- as.data.frame(matrix(rnorm(n * p), n, p))
y <- X[, 1] * beta + X[, 2] * beta + rnorm(n, sd = noise_sd)
data_ls[["Independent Features"]] <- data.frame(y = y, X)
# case 2: correlated non-signal features
p_cor <- 100
rho <- 0.9
sig <- matrix(rho, p_cor, p_cor)
X_cor <- MASS::mvrnorm(n, mu = rep(0, p_cor), Sigma = sig)
data_ls[["Correlated Non-signal Features"]] <- dplyr::bind_cols(
data.frame(y = y, X),
as.data.frame(X_cor) |> dplyr::rename_with(~ paste0("corr", .x))
)
# case 3: correlated signal features (one signal)
sig <- matrix(rho, p_cor + 1, p_cor + 1)
X_cor <- MASS::mvrnorm(n, mu = rep(0, p_cor + 1), Sigma = sig)
X[, 1] <- X_cor[, 1]
X_cor <- X_cor[, -1]
y <- X[, 1] * beta + X[, 2] * beta + rnorm(n, sd = noise_sd)
data_ls[["Correlated Signal Features (1 signal)"]] <- dplyr::bind_cols(
data.frame(y = y, X),
as.data.frame(X_cor) |> dplyr::rename_with(~ paste0("corr", .x))
)
# case 4: correlated signal features (many signal)
y <- X[, 1] * beta + X[, 2] * beta + rowSums(X_cor[, 1:4] * beta) + rnorm(n, sd = noise_sd)
data_ls[["Correlated Signal Features (many signal)"]] <- dplyr::bind_cols(
data.frame(y = y, X),
as.data.frame(X_cor) |> dplyr::rename_with(~ paste0("corr", .x))
)
for (sim_name in names(data_ls)) {
cat(sprintf("\n\n## %s {.tabset .tabset-pills .tabset-square}\n\n", sim_name))
data <- data_ls[[sim_name]]
y <- data |>
dplyr::pull(y)
X <- data |>
dplyr::select(-y)
# linear regression
lm_fit <- lm(y ~ ., data = data)
lm_vimp_df <- tibble::tibble(
var = names(summary(lm_fit)$coefficients[-1, 1]),
vimp = summary(lm_fit)$coefficients[-1, 1],
se = summary(lm_fit)$coefficients[-1, 2]
)
# LASSO regression
lasso_fit <- glmnet::cv.glmnet(
x = as.matrix(X),
y = y,
alpha = 1,
nfolds = 5
)
lasso_vimp_df <- tibble::tibble(
var = rownames(coef(lasso_fit, s = "lambda.min"))[-1],
vimp = as.matrix(coef(lasso_fit, s = "lambda.min"))[-1]
)
# ridge regression
ridge_fit <- glmnet::cv.glmnet(
x = as.matrix(X),
y = y,
alpha = 0,
nfolds = 5
)
ridge_vimp_df <- tibble::tibble(
var = rownames(coef(ridge_fit, s = "lambda.min"))[-1],
vimp = as.matrix(coef(ridge_fit, s = "lambda.min"))[-1]
)
# random forest (MDI)
rf_fit <- ranger::ranger(
data = data,
formula = y ~ .,
importance = "impurity"
)
rf_vimp_mdi_df <- tibble::tibble(
var = names(rf_fit$variable.importance),
vimp = rf_fit$variable.importance
)
# random forest (permutation)
rf_fit <- ranger::ranger(
data = data,
formula = y ~ .,
importance = "permutation"
)
rf_vimp_perm_df <- tibble::tibble(
var = names(rf_fit$variable.importance),
vimp = rf_fit$variable.importance
)
# random forest (feature occlusion)
oob_errs <- c() # using out-of-bag error as the metric for simplicity (should generally use held-out test set)
for (j in names(rf_fit$variable.importance)) {
X_loco_j <- X |>
dplyr::select(-tidyselect::all_of(j))
rf_fit_j <- ranger::ranger(
data = cbind(y = y, X_loco_j),
formula = y ~ .
)
oob_errs[j] <- rf_fit_j$prediction.error
}
rf_vimp_loco_df <- tibble::tibble(
var = names(rf_fit$variable.importance),
vimp = oob_errs - rf_fit$prediction.error
)
# random forest (shap)
rf_fit <- ranger::ranger(
data = data,
formula = y ~ .
)
pred_fun <- function(object, newdata) {
predict(object, newdata)$predictions
}
shap_values <- fastshap::explain(
object = rf_fit,
X = X,
pred_wrapper = pred_fun,
nsim = 10
)
rf_vimp_shap_df <- as.data.frame(abs(shap_values)) |>
dplyr::summarise(
dplyr::across(
tidyselect::everything(),
~ mean(.x)
)
) |>
tidyr::pivot_longer(
cols = tidyselect::everything(),
names_to = "var",
values_to = "vimp"
)
vimp_df <- list(
Linear = lm_vimp_df,
LASSO = lasso_vimp_df,
Ridge = ridge_vimp_df,
`RF (MDI)` = rf_vimp_mdi_df,
`RF (permute)` = rf_vimp_perm_df,
`RF (SHAP)` = rf_vimp_shap_df,
`RF (LOCO)` = rf_vimp_loco_df
) |>
dplyr::bind_rows(.id = "method") |>
dplyr::mutate(
method = forcats::fct_inorder(method),
color = dplyr::case_when(
var == "V1" ~ "Signal1",
var == "V2" ~ "Signal2",
stringr::str_detect(var, "corr") ~ "Correlated",
TRUE ~ "Other"
),
var = forcats::fct_inorder(var)
)
plt <- vimp_df |>
ggplot2::ggplot() +
ggplot2::aes(x = var, y = vimp, fill = color) +
ggplot2::geom_bar(stat = "identity") +
ggplot2::facet_wrap(~ method, ncol = 1, scales = "free_y") +
ggplot2::labs(x = "Feature", y = "Importance", fill = "") +
ggplot2::scale_fill_manual(
values = c(
Signal1 = "dodgerblue",
Signal2 = "darkgreen",
Correlated = "orange",
Other = "gray"
)
) +
vthemes::theme_vmodern(
x_text_angle = TRUE,
size_preset = "large"
)
fig_width <- dplyr::case_when(
ncol(data) > 20 ~ 20,
TRUE ~ 10
)
vthemes::subchunkify(
plt, i = subchunk_idx, fig_height = 16, fig_width = fig_width
)
subchunk_idx <- subchunk_idx + 1
fit_ls[[sim_name]] <- list(
Linear = lm_fit,
LASSO = lasso_fit,
Ridge = ridge_fit,
`RF` = rf_fit
)
vimp_ls[[sim_name]] <- vimp_df
}Independent Features
Correlated Non-signal Features
Correlated Signal Features (1 signal)
Correlated Signal Features (many signal)
Correlated Features: Ozone data
We will next investigate these different feature importance methods using a real-world dataset, namely the “Ozone” dataset from the mlbench R package. Using this dataset, we aim to predict the ozone readings using various meteorological features such as temperature, wind speed, humidity, air pressure, and other related measurements.
data("Ozone", package = "mlbench")
# clean data
data <- Ozone |>
dplyr::select(
ozone = V4,
temperature = V8,
inversion_height = V10,
pressure = V11,
visibility = V13,
millibar_pressure_height = V5,
humidity = V7,
inversion_temperature = V12,
wind = V6
)
na_samples <- apply(data, 1, function(x) any(is.na(x)))
data <- data[!na_samples, ]
y <- data |>
dplyr::pull(ozone)
X <- data |>
dplyr::select(-ozone)
plt <- data |>
tidyr::pivot_longer(
cols = tidyselect::everything(),
names_to = "feature",
values_to = "value"
) |>
ggplot2::ggplot() +
ggplot2::aes(x = value) +
ggplot2::facet_wrap(~ feature, scales = "free") +
ggplot2::geom_histogram(bins = 30) +
vthemes::theme_vmodern()
# linear regression
cat("\n\n## Linear Regression\n\n")Linear Regression
LASSO
lasso_fit <- glmnet::cv.glmnet(
x = as.matrix(X),
y = y,
alpha = 1
)
lasso_vimp_df <- tibble::tibble(
var = rownames(coef(lasso_fit, s = "lambda.min"))[-1],
vimp = as.matrix(coef(lasso_fit, s = "lambda.min"))[-1]
)
plot(lasso_fit)
lasso_path_df <- as.data.frame(as.matrix(lasso_fit$glmnet.fit$beta)) |>
setNames(lasso_fit$lambda) |>
tibble::rownames_to_column("Variable") |>
tidyr::pivot_longer(
cols = -Variable,
names_to = "Lambda",
values_to = "Coefficient"
) |>
dplyr::mutate(
Lambda = as.numeric(Lambda)
)
lasso_path_plt <- lasso_path_df |>
ggplot2::ggplot() +
ggplot2::aes(
x = log(Lambda),
y = Coefficient,
color = Variable,
group = Variable
) +
ggplot2::geom_line(linewidth = 1) +
ggplot2::geom_hline(yintercept = 0, color = "black", linetype = "dashed") +
vthemes::theme_vmodern(size_preset = "large")
vthemes::subchunkify(
plotly::ggplotly(lasso_path_plt), i = subchunk_idx,
fig_height = 6, fig_width = 10
)Ridge
ridge_fit <- glmnet::cv.glmnet(
x = as.matrix(X),
y = y,
alpha = 0,
nfolds = 5
)
ridge_vimp_df <- tibble::tibble(
var = rownames(coef(ridge_fit, s = "lambda.min"))[-1],
vimp = as.matrix(coef(ridge_fit, s = "lambda.min"))[-1]
)
plot(ridge_fit)
ridge_path_df <- as.data.frame(as.matrix(ridge_fit$glmnet.fit$beta)) |>
setNames(ridge_fit$lambda) |>
tibble::rownames_to_column("Variable") |>
tidyr::pivot_longer(
cols = -Variable,
names_to = "Lambda",
values_to = "Coefficient"
) |>
dplyr::mutate(
Lambda = as.numeric(Lambda)
)
ridge_path_plt <- ridge_path_df |>
ggplot2::ggplot() +
ggplot2::aes(
x = log(Lambda),
y = Coefficient,
color = Variable,
group = Variable
) +
ggplot2::geom_line(linewidth = 1) +
ggplot2::geom_hline(yintercept = 0, color = "black", linetype = "dashed") +
vthemes::theme_vmodern(size_preset = "large")
vthemes::subchunkify(
plotly::ggplotly(ridge_path_plt), i = subchunk_idx,
fig_height = 6, fig_width = 10
)subchunk_idx <- subchunk_idx + 1
# random forest (MDI)
rf_fit <- ranger::ranger(
data = data,
formula = ozone ~ .,
importance = "impurity"
)
rf_vimp_mdi_df <- tibble::tibble(
var = names(rf_fit$variable.importance),
vimp = rf_fit$variable.importance
)
# random forest (permutation)
rf_fit <- ranger::ranger(
data = data,
formula = ozone ~ .,
importance = "permutation"
)
rf_vimp_perm_df <- tibble::tibble(
var = names(rf_fit$variable.importance),
vimp = rf_fit$variable.importance
)
# random forest (feature occlusion)
oob_errs <- c() # using out-of-bag error as the metric for simplicity (should generally use held-out test set)
for (j in names(rf_fit$variable.importance)) {
X_loco_j <- X |>
dplyr::select(-tidyselect::all_of(j))
rf_fit_j <- ranger::ranger(
data = cbind(y = y, X_loco_j),
formula = y ~ .
)
oob_errs[j] <- rf_fit_j$prediction.error
}
rf_vimp_loco_df <- tibble::tibble(
var = names(rf_fit$variable.importance),
vimp = oob_errs - rf_fit$prediction.error
)
# random forest (shap)
rf_fit <- ranger::ranger(
data = data,
formula = ozone ~ .
)
pred_fun <- function(object, newdata) {
predict(object, newdata)$predictions
}
shap_values <- fastshap::explain(
object = rf_fit,
X = X,
pred_wrapper = pred_fun,
nsim = 10
)
rf_vimp_shap_df <- as.data.frame(abs(shap_values)) |>
dplyr::summarise(
dplyr::across(
tidyselect::everything(),
~ mean(.x)
)
) |>
tidyr::pivot_longer(
cols = tidyselect::everything(),
names_to = "var",
values_to = "vimp"
)
cat("\n\n## Summary\n\n")Summary
vimp_df <- list(
Linear = lm_vimp_df,
LASSO = lasso_vimp_df,
Ridge = ridge_vimp_df,
`RF (MDI)` = rf_vimp_mdi_df,
`RF (permute)` = rf_vimp_perm_df,
`RF (SHAP)` = rf_vimp_shap_df,
`RF (LOCO)` = rf_vimp_loco_df
) |>
dplyr::bind_rows(.id = "method") |>
dplyr::mutate(
method = forcats::fct_inorder(method)
)
plt <- vimp_df |>
ggplot2::ggplot() +
ggplot2::aes(x = var, y = vimp) +
ggplot2::geom_bar(stat = "identity") +
ggplot2::facet_wrap(~ method, ncol = 1, scales = "free_y") +
ggplot2::labs(x = "Feature", y = "Importance", fill = "") +
vthemes::theme_vmodern(
x_text_angle = TRUE,
size_preset = "large"
)
vthemes::subchunkify(
# Plot the heatmap
corrplot::corrplot(
cor(X), method = "color", order = "hclust",
col = colorRampPalette(c("blue", "white", "red"))(15),
addCoef.col = "black", # Add correlation coefficients
tl.col = "black", tl.srt = 45 # Rotate text labels
),
i = subchunk_idx, fig_height = 10, fig_width = 10
)
subchunk_idx <- subchunk_idx + 1
vthemes::subchunkify(
plt, i = subchunk_idx, fig_height = 16, fig_width = 10
)
LS0tCnRpdGxlOiAiQ29ycmVsYXRlZCBGZWF0dXJlcyIKYXV0aG9yOiAiVGlmZmFueSBUYW5nIgpkYXRlOiAiYHIgU3lzLkRhdGUoKWAiCm91dHB1dDogdnRoZW1lczo6dm1vZGVybgotLS0KCmBgYHtyIHNldHVwLCBpbmNsdWRlPUZBTFNFfQprbml0cjo6b3B0c19jaHVuayRzZXQoZWNobyA9IFRSVUUpCnNldC5zZWVkKDMzMSkKCnN1YmNodW5rX2lkeCA8LSAxCmBgYAoKIyBDb3JyZWxhdGVkIEZlYXR1cmVzIFNpbXVsYXRpb24gey50YWJzZXQgLnRhYnNldC12bW9kZXJufQoKPGRpdiBjbGFzcz0icGFuZWwgcGFuZWwtZGVmYXVsdCBwYWRkZWQtcGFuZWwiPgpXZSB3aWxsIGZpcnN0IGludmVzdGlnYXRlIGhvdyB2YXJpb3VzIGZlYXR1cmUgaW1wb3J0YW5jZSBtZXRob2RzIHRyZWF0IGNvcnJlbGF0ZWQgdmFyaWFibGVzIHVuZGVyIGRpZmZlcmVudCBzaW11bGF0aW9uIHNjZW5hcmlvcy4KPC9kaXY+CgpgYGB7ciByZXN1bHRzID0gImFzaXMifQpuIDwtIDUwMApwIDwtIDEwCmJldGEgPC0gMQpub2lzZV9zZCA8LSAxCmRhdGFfbHMgPC0gbGlzdCgpCnZpbXBfbHMgPC0gbGlzdCgpCmZpdF9scyA8LSBsaXN0KCkKCiMgY2FzZSAxOiBpbmRlcGVuZGVudCBmZWF0dXJlcwpYIDwtIGFzLmRhdGEuZnJhbWUobWF0cml4KHJub3JtKG4gKiBwKSwgbiwgcCkpCnkgPC0gWFssIDFdICogYmV0YSArIFhbLCAyXSAqIGJldGEgKyBybm9ybShuLCBzZCA9IG5vaXNlX3NkKQpkYXRhX2xzW1siSW5kZXBlbmRlbnQgRmVhdHVyZXMiXV0gPC0gZGF0YS5mcmFtZSh5ID0geSwgWCkKCiMgY2FzZSAyOiBjb3JyZWxhdGVkIG5vbi1zaWduYWwgZmVhdHVyZXMKcF9jb3IgPC0gMTAwCnJobyA8LSAwLjkKc2lnIDwtIG1hdHJpeChyaG8sIHBfY29yLCBwX2NvcikKWF9jb3IgPC0gTUFTUzo6bXZybm9ybShuLCBtdSA9IHJlcCgwLCBwX2NvciksIFNpZ21hID0gc2lnKQpkYXRhX2xzW1siQ29ycmVsYXRlZCBOb24tc2lnbmFsIEZlYXR1cmVzIl1dIDwtIGRwbHlyOjpiaW5kX2NvbHMoCiAgZGF0YS5mcmFtZSh5ID0geSwgWCksIAogIGFzLmRhdGEuZnJhbWUoWF9jb3IpIHw+IGRwbHlyOjpyZW5hbWVfd2l0aCh+IHBhc3RlMCgiY29yciIsIC54KSkKKQoKIyBjYXNlIDM6IGNvcnJlbGF0ZWQgc2lnbmFsIGZlYXR1cmVzIChvbmUgc2lnbmFsKQpzaWcgPC0gbWF0cml4KHJobywgcF9jb3IgKyAxLCBwX2NvciArIDEpClhfY29yIDwtIE1BU1M6Om12cm5vcm0obiwgbXUgPSByZXAoMCwgcF9jb3IgKyAxKSwgU2lnbWEgPSBzaWcpClhbLCAxXSA8LSBYX2NvclssIDFdClhfY29yIDwtIFhfY29yWywgLTFdCnkgPC0gWFssIDFdICogYmV0YSArIFhbLCAyXSAqIGJldGEgKyBybm9ybShuLCBzZCA9IG5vaXNlX3NkKQpkYXRhX2xzW1siQ29ycmVsYXRlZCBTaWduYWwgRmVhdHVyZXMgKDEgc2lnbmFsKSJdXSA8LSBkcGx5cjo6YmluZF9jb2xzKAogIGRhdGEuZnJhbWUoeSA9IHksIFgpLCAKICBhcy5kYXRhLmZyYW1lKFhfY29yKSB8PiBkcGx5cjo6cmVuYW1lX3dpdGgofiBwYXN0ZTAoImNvcnIiLCAueCkpCikKCiMgY2FzZSA0OiBjb3JyZWxhdGVkIHNpZ25hbCBmZWF0dXJlcyAobWFueSBzaWduYWwpCnkgPC0gWFssIDFdICogYmV0YSArIFhbLCAyXSAqIGJldGEgKyByb3dTdW1zKFhfY29yWywgMTo0XSAqIGJldGEpICsgcm5vcm0obiwgc2QgPSBub2lzZV9zZCkKZGF0YV9sc1tbIkNvcnJlbGF0ZWQgU2lnbmFsIEZlYXR1cmVzIChtYW55IHNpZ25hbCkiXV0gPC0gZHBseXI6OmJpbmRfY29scygKICBkYXRhLmZyYW1lKHkgPSB5LCBYKSwgCiAgYXMuZGF0YS5mcmFtZShYX2NvcikgfD4gZHBseXI6OnJlbmFtZV93aXRoKH4gcGFzdGUwKCJjb3JyIiwgLngpKQopCgpmb3IgKHNpbV9uYW1lIGluIG5hbWVzKGRhdGFfbHMpKSB7CiAgY2F0KHNwcmludGYoIlxuXG4jIyAlcyB7LnRhYnNldCAudGFic2V0LXBpbGxzIC50YWJzZXQtc3F1YXJlfVxuXG4iLCBzaW1fbmFtZSkpCiAgCiAgZGF0YSA8LSBkYXRhX2xzW1tzaW1fbmFtZV1dCiAgeSA8LSBkYXRhIHw+CiAgICBkcGx5cjo6cHVsbCh5KQogIFggPC0gZGF0YSB8PiAKICAgIGRwbHlyOjpzZWxlY3QoLXkpCiAgCiAgIyBsaW5lYXIgcmVncmVzc2lvbgogIGxtX2ZpdCA8LSBsbSh5IH4gLiwgZGF0YSA9IGRhdGEpCiAgbG1fdmltcF9kZiA8LSB0aWJibGU6OnRpYmJsZSgKICAgIHZhciA9IG5hbWVzKHN1bW1hcnkobG1fZml0KSRjb2VmZmljaWVudHNbLTEsIDFdKSwKICAgIHZpbXAgPSBzdW1tYXJ5KGxtX2ZpdCkkY29lZmZpY2llbnRzWy0xLCAxXSwKICAgIHNlID0gc3VtbWFyeShsbV9maXQpJGNvZWZmaWNpZW50c1stMSwgMl0KICApCiAgCiAgIyBMQVNTTyByZWdyZXNzaW9uCiAgbGFzc29fZml0IDwtIGdsbW5ldDo6Y3YuZ2xtbmV0KAogICAgeCA9IGFzLm1hdHJpeChYKSwKICAgIHkgPSB5LAogICAgYWxwaGEgPSAxLCAKICAgIG5mb2xkcyA9IDUKICApCiAgbGFzc29fdmltcF9kZiA8LSB0aWJibGU6OnRpYmJsZSgKICAgIHZhciA9IHJvd25hbWVzKGNvZWYobGFzc29fZml0LCBzID0gImxhbWJkYS5taW4iKSlbLTFdLAogICAgdmltcCA9IGFzLm1hdHJpeChjb2VmKGxhc3NvX2ZpdCwgcyA9ICJsYW1iZGEubWluIikpWy0xXQogICkKICAKICAjIHJpZGdlIHJlZ3Jlc3Npb24KICByaWRnZV9maXQgPC0gZ2xtbmV0Ojpjdi5nbG1uZXQoCiAgICB4ID0gYXMubWF0cml4KFgpLAogICAgeSA9IHksCiAgICBhbHBoYSA9IDAsIAogICAgbmZvbGRzID0gNQogICkKICByaWRnZV92aW1wX2RmIDwtIHRpYmJsZTo6dGliYmxlKAogICAgdmFyID0gcm93bmFtZXMoY29lZihyaWRnZV9maXQsIHMgPSAibGFtYmRhLm1pbiIpKVstMV0sCiAgICB2aW1wID0gYXMubWF0cml4KGNvZWYocmlkZ2VfZml0LCBzID0gImxhbWJkYS5taW4iKSlbLTFdCiAgKQogIAogICMgcmFuZG9tIGZvcmVzdCAoTURJKQogIHJmX2ZpdCA8LSByYW5nZXI6OnJhbmdlcigKICAgIGRhdGEgPSBkYXRhLAogICAgZm9ybXVsYSA9IHkgfiAuLAogICAgaW1wb3J0YW5jZSA9ICJpbXB1cml0eSIKICApCiAgcmZfdmltcF9tZGlfZGYgPC0gdGliYmxlOjp0aWJibGUoCiAgICB2YXIgPSBuYW1lcyhyZl9maXQkdmFyaWFibGUuaW1wb3J0YW5jZSksCiAgICB2aW1wID0gcmZfZml0JHZhcmlhYmxlLmltcG9ydGFuY2UKICApCiAgCiAgIyByYW5kb20gZm9yZXN0IChwZXJtdXRhdGlvbikKICByZl9maXQgPC0gcmFuZ2VyOjpyYW5nZXIoCiAgICBkYXRhID0gZGF0YSwKICAgIGZvcm11bGEgPSB5IH4gLiwKICAgIGltcG9ydGFuY2UgPSAicGVybXV0YXRpb24iCiAgKQogIHJmX3ZpbXBfcGVybV9kZiA8LSB0aWJibGU6OnRpYmJsZSgKICAgIHZhciA9IG5hbWVzKHJmX2ZpdCR2YXJpYWJsZS5pbXBvcnRhbmNlKSwKICAgIHZpbXAgPSByZl9maXQkdmFyaWFibGUuaW1wb3J0YW5jZQogICkKICAKICAjIHJhbmRvbSBmb3Jlc3QgKGZlYXR1cmUgb2NjbHVzaW9uKQogIG9vYl9lcnJzIDwtIGMoKSAgIyB1c2luZyBvdXQtb2YtYmFnIGVycm9yIGFzIHRoZSBtZXRyaWMgZm9yIHNpbXBsaWNpdHkgKHNob3VsZCBnZW5lcmFsbHkgdXNlIGhlbGQtb3V0IHRlc3Qgc2V0KQogIGZvciAoaiBpbiBuYW1lcyhyZl9maXQkdmFyaWFibGUuaW1wb3J0YW5jZSkpIHsKICAgIFhfbG9jb19qIDwtIFggfD4KICAgICAgZHBseXI6OnNlbGVjdCgtdGlkeXNlbGVjdDo6YWxsX29mKGopKQogICAgcmZfZml0X2ogPC0gcmFuZ2VyOjpyYW5nZXIoCiAgICAgIGRhdGEgPSBjYmluZCh5ID0geSwgWF9sb2NvX2opLAogICAgICBmb3JtdWxhID0geSB+IC4KICAgICkKICAgIG9vYl9lcnJzW2pdIDwtIHJmX2ZpdF9qJHByZWRpY3Rpb24uZXJyb3IKICB9CiAgcmZfdmltcF9sb2NvX2RmIDwtIHRpYmJsZTo6dGliYmxlKAogICAgdmFyID0gbmFtZXMocmZfZml0JHZhcmlhYmxlLmltcG9ydGFuY2UpLAogICAgdmltcCA9IG9vYl9lcnJzIC0gcmZfZml0JHByZWRpY3Rpb24uZXJyb3IKICApCiAgCiAgIyByYW5kb20gZm9yZXN0IChzaGFwKQogIHJmX2ZpdCA8LSByYW5nZXI6OnJhbmdlcigKICAgIGRhdGEgPSBkYXRhLAogICAgZm9ybXVsYSA9IHkgfiAuCiAgKQogIHByZWRfZnVuIDwtIGZ1bmN0aW9uKG9iamVjdCwgbmV3ZGF0YSkgewogICAgcHJlZGljdChvYmplY3QsIG5ld2RhdGEpJHByZWRpY3Rpb25zCiAgfQogIHNoYXBfdmFsdWVzIDwtIGZhc3RzaGFwOjpleHBsYWluKAogICAgb2JqZWN0ID0gcmZfZml0LAogICAgWCA9IFgsCiAgICBwcmVkX3dyYXBwZXIgPSBwcmVkX2Z1biwKICAgIG5zaW0gPSAxMAogICkKICByZl92aW1wX3NoYXBfZGYgPC0gYXMuZGF0YS5mcmFtZShhYnMoc2hhcF92YWx1ZXMpKSB8PiAKICAgIGRwbHlyOjpzdW1tYXJpc2UoCiAgICAgIGRwbHlyOjphY3Jvc3MoCiAgICAgICAgdGlkeXNlbGVjdDo6ZXZlcnl0aGluZygpLAogICAgICAgIH4gbWVhbigueCkKICAgICAgKQogICAgKSB8PgogICAgdGlkeXI6OnBpdm90X2xvbmdlcigKICAgICAgY29scyA9IHRpZHlzZWxlY3Q6OmV2ZXJ5dGhpbmcoKSwKICAgICAgbmFtZXNfdG8gPSAidmFyIiwKICAgICAgdmFsdWVzX3RvID0gInZpbXAiCiAgICApCiAgCiAgdmltcF9kZiA8LSBsaXN0KAogICAgTGluZWFyID0gbG1fdmltcF9kZiwKICAgIExBU1NPID0gbGFzc29fdmltcF9kZiwKICAgIFJpZGdlID0gcmlkZ2VfdmltcF9kZiwKICAgIGBSRiAoTURJKWAgPSByZl92aW1wX21kaV9kZiwKICAgIGBSRiAocGVybXV0ZSlgID0gcmZfdmltcF9wZXJtX2RmLAogICAgYFJGIChTSEFQKWAgPSByZl92aW1wX3NoYXBfZGYsCiAgICBgUkYgKExPQ08pYCA9IHJmX3ZpbXBfbG9jb19kZgogICkgfD4KICAgIGRwbHlyOjpiaW5kX3Jvd3MoLmlkID0gIm1ldGhvZCIpIHw+IAogICAgZHBseXI6Om11dGF0ZSgKICAgICAgbWV0aG9kID0gZm9yY2F0czo6ZmN0X2lub3JkZXIobWV0aG9kKSwKICAgICAgY29sb3IgPSBkcGx5cjo6Y2FzZV93aGVuKAogICAgICAgIHZhciA9PSAiVjEiIH4gIlNpZ25hbDEiLAogICAgICAgIHZhciA9PSAiVjIiIH4gIlNpZ25hbDIiLAogICAgICAgIHN0cmluZ3I6OnN0cl9kZXRlY3QodmFyLCAiY29yciIpIH4gIkNvcnJlbGF0ZWQiLAogICAgICAgIFRSVUUgfiAiT3RoZXIiCiAgICAgICksCiAgICAgIHZhciA9IGZvcmNhdHM6OmZjdF9pbm9yZGVyKHZhcikKICAgICkKICAKICBwbHQgPC0gdmltcF9kZiB8PiAKICAgIGdncGxvdDI6OmdncGxvdCgpICsKICAgIGdncGxvdDI6OmFlcyh4ID0gdmFyLCB5ID0gdmltcCwgZmlsbCA9IGNvbG9yKSArCiAgICBnZ3Bsb3QyOjpnZW9tX2JhcihzdGF0ID0gImlkZW50aXR5IikgKwogICAgZ2dwbG90Mjo6ZmFjZXRfd3JhcCh+IG1ldGhvZCwgbmNvbCA9IDEsIHNjYWxlcyA9ICJmcmVlX3kiKSArCiAgICBnZ3Bsb3QyOjpsYWJzKHggPSAiRmVhdHVyZSIsIHkgPSAiSW1wb3J0YW5jZSIsIGZpbGwgPSAiIikgKwogICAgZ2dwbG90Mjo6c2NhbGVfZmlsbF9tYW51YWwoCiAgICAgIHZhbHVlcyA9IGMoCiAgICAgICAgU2lnbmFsMSA9ICJkb2RnZXJibHVlIiwKICAgICAgICBTaWduYWwyID0gImRhcmtncmVlbiIsCiAgICAgICAgQ29ycmVsYXRlZCA9ICJvcmFuZ2UiLAogICAgICAgIE90aGVyID0gImdyYXkiCiAgICAgICkKICAgICkgKwogICAgdnRoZW1lczo6dGhlbWVfdm1vZGVybigKICAgICAgeF90ZXh0X2FuZ2xlID0gVFJVRSwKICAgICAgc2l6ZV9wcmVzZXQgPSAibGFyZ2UiCiAgICApCiAgZmlnX3dpZHRoIDwtIGRwbHlyOjpjYXNlX3doZW4oCiAgICBuY29sKGRhdGEpID4gMjAgfiAyMCwKICAgIFRSVUUgfiAxMAogICkKICB2dGhlbWVzOjpzdWJjaHVua2lmeSgKICAgIHBsdCwgaSA9IHN1YmNodW5rX2lkeCwgZmlnX2hlaWdodCA9IDE2LCBmaWdfd2lkdGggPSBmaWdfd2lkdGgKICApCiAgc3ViY2h1bmtfaWR4IDwtIHN1YmNodW5rX2lkeCArIDEKICAKICBmaXRfbHNbW3NpbV9uYW1lXV0gPC0gbGlzdCgKICAgIExpbmVhciA9IGxtX2ZpdCwKICAgIExBU1NPID0gbGFzc29fZml0LAogICAgUmlkZ2UgPSByaWRnZV9maXQsCiAgICBgUkZgID0gcmZfZml0CiAgKQogIHZpbXBfbHNbW3NpbV9uYW1lXV0gPC0gdmltcF9kZgp9CmBgYAoKIyBDb3JyZWxhdGVkIEZlYXR1cmVzOiBPem9uZSBkYXRhIHsudGFic2V0IC50YWJzZXQtdm1vZGVybn0KCjxkaXYgY2xhc3M9InBhbmVsIHBhbmVsLWRlZmF1bHQgcGFkZGVkLXBhbmVsIj4KV2Ugd2lsbCBuZXh0IGludmVzdGlnYXRlIHRoZXNlIGRpZmZlcmVudCBmZWF0dXJlIGltcG9ydGFuY2UgbWV0aG9kcyB1c2luZyBhIHJlYWwtd29ybGQgZGF0YXNldCwgbmFtZWx5IHRoZSAiT3pvbmUiIGRhdGFzZXQgZnJvbSB0aGUgYG1sYmVuY2hgIFIgcGFja2FnZS4gVXNpbmcgdGhpcyBkYXRhc2V0LCB3ZSBhaW0gdG8gcHJlZGljdCB0aGUgb3pvbmUgcmVhZGluZ3MgdXNpbmcgdmFyaW91cyBtZXRlb3JvbG9naWNhbCBmZWF0dXJlcyBzdWNoIGFzIHRlbXBlcmF0dXJlLCB3aW5kIHNwZWVkLCBodW1pZGl0eSwgYWlyIHByZXNzdXJlLCBhbmQgb3RoZXIgcmVsYXRlZCBtZWFzdXJlbWVudHMuCjwvZGl2PgoKYGBge3IgcmVzdWx0cyA9ICJhc2lzIn0KZGF0YSgiT3pvbmUiLCBwYWNrYWdlID0gIm1sYmVuY2giKQoKIyBjbGVhbiBkYXRhCmRhdGEgPC0gT3pvbmUgfD4gCiAgZHBseXI6OnNlbGVjdCgKICAgIG96b25lID0gVjQsCiAgICB0ZW1wZXJhdHVyZSA9IFY4LAogICAgaW52ZXJzaW9uX2hlaWdodCA9IFYxMCwKICAgIHByZXNzdXJlID0gVjExLAogICAgdmlzaWJpbGl0eSA9IFYxMywKICAgIG1pbGxpYmFyX3ByZXNzdXJlX2hlaWdodCA9IFY1LAogICAgaHVtaWRpdHkgPSBWNywKICAgIGludmVyc2lvbl90ZW1wZXJhdHVyZSA9IFYxMiwKICAgIHdpbmQgPSBWNgogICkKbmFfc2FtcGxlcyA8LSBhcHBseShkYXRhLCAxLCBmdW5jdGlvbih4KSBhbnkoaXMubmEoeCkpKQpkYXRhIDwtIGRhdGFbIW5hX3NhbXBsZXMsIF0KCnkgPC0gZGF0YSB8PgogIGRwbHlyOjpwdWxsKG96b25lKQpYIDwtIGRhdGEgfD4gCiAgZHBseXI6OnNlbGVjdCgtb3pvbmUpCgpwbHQgPC0gZGF0YSB8PiAKICB0aWR5cjo6cGl2b3RfbG9uZ2VyKAogICAgY29scyA9IHRpZHlzZWxlY3Q6OmV2ZXJ5dGhpbmcoKSwgCiAgICBuYW1lc190byA9ICJmZWF0dXJlIiwgCiAgICB2YWx1ZXNfdG8gPSAidmFsdWUiCiAgKSB8PiAKICBnZ3Bsb3QyOjpnZ3Bsb3QoKSArCiAgZ2dwbG90Mjo6YWVzKHggPSB2YWx1ZSkgKwogIGdncGxvdDI6OmZhY2V0X3dyYXAofiBmZWF0dXJlLCBzY2FsZXMgPSAiZnJlZSIpICsKICBnZ3Bsb3QyOjpnZW9tX2hpc3RvZ3JhbShiaW5zID0gMzApICsKICB2dGhlbWVzOjp0aGVtZV92bW9kZXJuKCkKCiMgbGluZWFyIHJlZ3Jlc3Npb24KY2F0KCJcblxuIyMgTGluZWFyIFJlZ3Jlc3Npb25cblxuIikKbG1fZml0IDwtIGxtKG96b25lIH4gLiwgZGF0YSA9IGRhdGEpCmxtX3ZpbXBfZGYgPC0gdGliYmxlOjp0aWJibGUoCiAgdmFyID0gbmFtZXMoc3VtbWFyeShsbV9maXQpJGNvZWZmaWNpZW50c1stMSwgMV0pLAogIHZpbXAgPSBzdW1tYXJ5KGxtX2ZpdCkkY29lZmZpY2llbnRzWy0xLCAxXSwKICBzZSA9IHN1bW1hcnkobG1fZml0KSRjb2VmZmljaWVudHNbLTEsIDJdCikKYnJvb206OnRpZHkobG1fZml0KSB8PiAKICB2dGhlbWVzOjpwcmV0dHlfRFQoKQoKIyBMQVNTTyByZWdyZXNzaW9uCmNhdCgiXG5cbiMjIExBU1NPXG5cbiIpCmxhc3NvX2ZpdCA8LSBnbG1uZXQ6OmN2LmdsbW5ldCgKICB4ID0gYXMubWF0cml4KFgpLAogIHkgPSB5LAogIGFscGhhID0gMQopCmxhc3NvX3ZpbXBfZGYgPC0gdGliYmxlOjp0aWJibGUoCiAgdmFyID0gcm93bmFtZXMoY29lZihsYXNzb19maXQsIHMgPSAibGFtYmRhLm1pbiIpKVstMV0sCiAgdmltcCA9IGFzLm1hdHJpeChjb2VmKGxhc3NvX2ZpdCwgcyA9ICJsYW1iZGEubWluIikpWy0xXQopCnBsb3QobGFzc29fZml0KQpsYXNzb19wYXRoX2RmIDwtIGFzLmRhdGEuZnJhbWUoYXMubWF0cml4KGxhc3NvX2ZpdCRnbG1uZXQuZml0JGJldGEpKSB8PgogIHNldE5hbWVzKGxhc3NvX2ZpdCRsYW1iZGEpIHw+IAogIHRpYmJsZTo6cm93bmFtZXNfdG9fY29sdW1uKCJWYXJpYWJsZSIpIHw+IAogIHRpZHlyOjpwaXZvdF9sb25nZXIoCiAgICBjb2xzID0gLVZhcmlhYmxlLCAKICAgIG5hbWVzX3RvID0gIkxhbWJkYSIsIAogICAgdmFsdWVzX3RvID0gIkNvZWZmaWNpZW50IgogICkgfD4gCiAgZHBseXI6Om11dGF0ZSgKICAgIExhbWJkYSA9IGFzLm51bWVyaWMoTGFtYmRhKQogICkKbGFzc29fcGF0aF9wbHQgPC0gbGFzc29fcGF0aF9kZiB8PiAKICBnZ3Bsb3QyOjpnZ3Bsb3QoKSArCiAgZ2dwbG90Mjo6YWVzKAogICAgeCA9IGxvZyhMYW1iZGEpLAogICAgeSA9IENvZWZmaWNpZW50LAogICAgY29sb3IgPSBWYXJpYWJsZSwKICAgIGdyb3VwID0gVmFyaWFibGUKICApICsKICBnZ3Bsb3QyOjpnZW9tX2xpbmUobGluZXdpZHRoID0gMSkgKwogIGdncGxvdDI6Omdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGNvbG9yID0gImJsYWNrIiwgbGluZXR5cGUgPSAiZGFzaGVkIikgKwogIHZ0aGVtZXM6OnRoZW1lX3Ztb2Rlcm4oc2l6ZV9wcmVzZXQgPSAibGFyZ2UiKQp2dGhlbWVzOjpzdWJjaHVua2lmeSgKICBwbG90bHk6OmdncGxvdGx5KGxhc3NvX3BhdGhfcGx0KSwgaSA9IHN1YmNodW5rX2lkeCwKICBmaWdfaGVpZ2h0ID0gNiwgZmlnX3dpZHRoID0gMTAKKQpzdWJjaHVua19pZHggPC0gc3ViY2h1bmtfaWR4ICsgMQoKIyByaWRnZSByZWdyZXNzaW9uCmNhdCgiXG5cbiMjIFJpZGdlXG5cbiIpCnJpZGdlX2ZpdCA8LSBnbG1uZXQ6OmN2LmdsbW5ldCgKICB4ID0gYXMubWF0cml4KFgpLAogIHkgPSB5LAogIGFscGhhID0gMCwgCiAgbmZvbGRzID0gNQopCnJpZGdlX3ZpbXBfZGYgPC0gdGliYmxlOjp0aWJibGUoCiAgdmFyID0gcm93bmFtZXMoY29lZihyaWRnZV9maXQsIHMgPSAibGFtYmRhLm1pbiIpKVstMV0sCiAgdmltcCA9IGFzLm1hdHJpeChjb2VmKHJpZGdlX2ZpdCwgcyA9ICJsYW1iZGEubWluIikpWy0xXQopCnBsb3QocmlkZ2VfZml0KQpyaWRnZV9wYXRoX2RmIDwtIGFzLmRhdGEuZnJhbWUoYXMubWF0cml4KHJpZGdlX2ZpdCRnbG1uZXQuZml0JGJldGEpKSB8PgogIHNldE5hbWVzKHJpZGdlX2ZpdCRsYW1iZGEpIHw+IAogIHRpYmJsZTo6cm93bmFtZXNfdG9fY29sdW1uKCJWYXJpYWJsZSIpIHw+IAogIHRpZHlyOjpwaXZvdF9sb25nZXIoCiAgICBjb2xzID0gLVZhcmlhYmxlLCAKICAgIG5hbWVzX3RvID0gIkxhbWJkYSIsIAogICAgdmFsdWVzX3RvID0gIkNvZWZmaWNpZW50IgogICkgfD4gCiAgZHBseXI6Om11dGF0ZSgKICAgIExhbWJkYSA9IGFzLm51bWVyaWMoTGFtYmRhKQogICkKcmlkZ2VfcGF0aF9wbHQgPC0gcmlkZ2VfcGF0aF9kZiB8PiAKICBnZ3Bsb3QyOjpnZ3Bsb3QoKSArCiAgZ2dwbG90Mjo6YWVzKAogICAgeCA9IGxvZyhMYW1iZGEpLAogICAgeSA9IENvZWZmaWNpZW50LAogICAgY29sb3IgPSBWYXJpYWJsZSwKICAgIGdyb3VwID0gVmFyaWFibGUKICApICsKICBnZ3Bsb3QyOjpnZW9tX2xpbmUobGluZXdpZHRoID0gMSkgKwogIGdncGxvdDI6Omdlb21faGxpbmUoeWludGVyY2VwdCA9IDAsIGNvbG9yID0gImJsYWNrIiwgbGluZXR5cGUgPSAiZGFzaGVkIikgKwogIHZ0aGVtZXM6OnRoZW1lX3Ztb2Rlcm4oc2l6ZV9wcmVzZXQgPSAibGFyZ2UiKQp2dGhlbWVzOjpzdWJjaHVua2lmeSgKICBwbG90bHk6OmdncGxvdGx5KHJpZGdlX3BhdGhfcGx0KSwgaSA9IHN1YmNodW5rX2lkeCwKICBmaWdfaGVpZ2h0ID0gNiwgZmlnX3dpZHRoID0gMTAKKQpzdWJjaHVua19pZHggPC0gc3ViY2h1bmtfaWR4ICsgMQoKIyByYW5kb20gZm9yZXN0IChNREkpCnJmX2ZpdCA8LSByYW5nZXI6OnJhbmdlcigKICBkYXRhID0gZGF0YSwKICBmb3JtdWxhID0gb3pvbmUgfiAuLAogIGltcG9ydGFuY2UgPSAiaW1wdXJpdHkiCikKcmZfdmltcF9tZGlfZGYgPC0gdGliYmxlOjp0aWJibGUoCiAgdmFyID0gbmFtZXMocmZfZml0JHZhcmlhYmxlLmltcG9ydGFuY2UpLAogIHZpbXAgPSByZl9maXQkdmFyaWFibGUuaW1wb3J0YW5jZQopCgojIHJhbmRvbSBmb3Jlc3QgKHBlcm11dGF0aW9uKQpyZl9maXQgPC0gcmFuZ2VyOjpyYW5nZXIoCiAgZGF0YSA9IGRhdGEsCiAgZm9ybXVsYSA9IG96b25lIH4gLiwKICBpbXBvcnRhbmNlID0gInBlcm11dGF0aW9uIgopCnJmX3ZpbXBfcGVybV9kZiA8LSB0aWJibGU6OnRpYmJsZSgKICB2YXIgPSBuYW1lcyhyZl9maXQkdmFyaWFibGUuaW1wb3J0YW5jZSksCiAgdmltcCA9IHJmX2ZpdCR2YXJpYWJsZS5pbXBvcnRhbmNlCikKCiMgcmFuZG9tIGZvcmVzdCAoZmVhdHVyZSBvY2NsdXNpb24pCm9vYl9lcnJzIDwtIGMoKSAgIyB1c2luZyBvdXQtb2YtYmFnIGVycm9yIGFzIHRoZSBtZXRyaWMgZm9yIHNpbXBsaWNpdHkgKHNob3VsZCBnZW5lcmFsbHkgdXNlIGhlbGQtb3V0IHRlc3Qgc2V0KQpmb3IgKGogaW4gbmFtZXMocmZfZml0JHZhcmlhYmxlLmltcG9ydGFuY2UpKSB7CiAgWF9sb2NvX2ogPC0gWCB8PgogICAgZHBseXI6OnNlbGVjdCgtdGlkeXNlbGVjdDo6YWxsX29mKGopKQogIHJmX2ZpdF9qIDwtIHJhbmdlcjo6cmFuZ2VyKAogICAgZGF0YSA9IGNiaW5kKHkgPSB5LCBYX2xvY29faiksCiAgICBmb3JtdWxhID0geSB+IC4KICApCiAgb29iX2VycnNbal0gPC0gcmZfZml0X2okcHJlZGljdGlvbi5lcnJvcgp9CnJmX3ZpbXBfbG9jb19kZiA8LSB0aWJibGU6OnRpYmJsZSgKICB2YXIgPSBuYW1lcyhyZl9maXQkdmFyaWFibGUuaW1wb3J0YW5jZSksCiAgdmltcCA9IG9vYl9lcnJzIC0gcmZfZml0JHByZWRpY3Rpb24uZXJyb3IKKQoKIyByYW5kb20gZm9yZXN0IChzaGFwKQpyZl9maXQgPC0gcmFuZ2VyOjpyYW5nZXIoCiAgZGF0YSA9IGRhdGEsCiAgZm9ybXVsYSA9IG96b25lIH4gLgopCnByZWRfZnVuIDwtIGZ1bmN0aW9uKG9iamVjdCwgbmV3ZGF0YSkgewogIHByZWRpY3Qob2JqZWN0LCBuZXdkYXRhKSRwcmVkaWN0aW9ucwp9CnNoYXBfdmFsdWVzIDwtIGZhc3RzaGFwOjpleHBsYWluKAogIG9iamVjdCA9IHJmX2ZpdCwKICBYID0gWCwKICBwcmVkX3dyYXBwZXIgPSBwcmVkX2Z1biwKICBuc2ltID0gMTAKKQpyZl92aW1wX3NoYXBfZGYgPC0gYXMuZGF0YS5mcmFtZShhYnMoc2hhcF92YWx1ZXMpKSB8PiAKICBkcGx5cjo6c3VtbWFyaXNlKAogICAgZHBseXI6OmFjcm9zcygKICAgICAgdGlkeXNlbGVjdDo6ZXZlcnl0aGluZygpLAogICAgICB+IG1lYW4oLngpCiAgICApCiAgKSB8PgogIHRpZHlyOjpwaXZvdF9sb25nZXIoCiAgICBjb2xzID0gdGlkeXNlbGVjdDo6ZXZlcnl0aGluZygpLAogICAgbmFtZXNfdG8gPSAidmFyIiwKICAgIHZhbHVlc190byA9ICJ2aW1wIgogICkKCmNhdCgiXG5cbiMjIFN1bW1hcnlcblxuIikKdmltcF9kZiA8LSBsaXN0KAogIExpbmVhciA9IGxtX3ZpbXBfZGYsCiAgTEFTU08gPSBsYXNzb192aW1wX2RmLAogIFJpZGdlID0gcmlkZ2VfdmltcF9kZiwKICBgUkYgKE1ESSlgID0gcmZfdmltcF9tZGlfZGYsCiAgYFJGIChwZXJtdXRlKWAgPSByZl92aW1wX3Blcm1fZGYsCiAgYFJGIChTSEFQKWAgPSByZl92aW1wX3NoYXBfZGYsCiAgYFJGIChMT0NPKWAgPSByZl92aW1wX2xvY29fZGYKKSB8PgogIGRwbHlyOjpiaW5kX3Jvd3MoLmlkID0gIm1ldGhvZCIpIHw+IAogIGRwbHlyOjptdXRhdGUoCiAgICBtZXRob2QgPSBmb3JjYXRzOjpmY3RfaW5vcmRlcihtZXRob2QpCiAgKQoKcGx0IDwtIHZpbXBfZGYgfD4gCiAgZ2dwbG90Mjo6Z2dwbG90KCkgKwogIGdncGxvdDI6OmFlcyh4ID0gdmFyLCB5ID0gdmltcCkgKwogIGdncGxvdDI6Omdlb21fYmFyKHN0YXQgPSAiaWRlbnRpdHkiKSArCiAgZ2dwbG90Mjo6ZmFjZXRfd3JhcCh+IG1ldGhvZCwgbmNvbCA9IDEsIHNjYWxlcyA9ICJmcmVlX3kiKSArCiAgZ2dwbG90Mjo6bGFicyh4ID0gIkZlYXR1cmUiLCB5ID0gIkltcG9ydGFuY2UiLCBmaWxsID0gIiIpICsKICB2dGhlbWVzOjp0aGVtZV92bW9kZXJuKAogICAgeF90ZXh0X2FuZ2xlID0gVFJVRSwKICAgIHNpemVfcHJlc2V0ID0gImxhcmdlIgogICkKCnZ0aGVtZXM6OnN1YmNodW5raWZ5KAogICMgUGxvdCB0aGUgaGVhdG1hcAogIGNvcnJwbG90Ojpjb3JycGxvdCgKICAgIGNvcihYKSwgbWV0aG9kID0gImNvbG9yIiwgb3JkZXIgPSAiaGNsdXN0IiwKICAgIGNvbCA9IGNvbG9yUmFtcFBhbGV0dGUoYygiYmx1ZSIsICJ3aGl0ZSIsICJyZWQiKSkoMTUpLAogICAgYWRkQ29lZi5jb2wgPSAiYmxhY2siLCAjIEFkZCBjb3JyZWxhdGlvbiBjb2VmZmljaWVudHMKICAgIHRsLmNvbCA9ICJibGFjayIsIHRsLnNydCA9IDQ1ICMgUm90YXRlIHRleHQgbGFiZWxzCiAgKSwKICBpID0gc3ViY2h1bmtfaWR4LCBmaWdfaGVpZ2h0ID0gMTAsIGZpZ193aWR0aCA9IDEwCikKc3ViY2h1bmtfaWR4IDwtIHN1YmNodW5rX2lkeCArIDEKdnRoZW1lczo6c3ViY2h1bmtpZnkoCiAgcGx0LCBpID0gc3ViY2h1bmtfaWR4LCBmaWdfaGVpZ2h0ID0gMTYsIGZpZ193aWR0aCA9IDEwCikKc3ViY2h1bmtfaWR4IDwtIHN1YmNodW5rX2lkeCArIDEKYGBgCgoK